@echo off
REM 定义保存目录
set "DOWNLOAD_DIR=%CD%\downloaded_files"
set "LOG_FILE=spider.log"
set "LINKS_FILE=file_links.txt"

REM 创建保存目录
mkdir "%DOWNLOAD_DIR%"

REM 第一步：模拟抓取所有链接并保存日志
echo Crawling the webpage to gather all links...
wget --spider -r -nd -A "*.hic,*.fasta,*.assembly" -o "%LOG_FILE%" "https://sid.erda.dk/cgi-sid/ls.py?share_id=GxrFNGtykS"

REM 第二步：从日志中提取文件链接
echo Extracting file links from log...
findstr /i "http://" "%LOG_FILE%" | findstr /r "\.pdf$ \.zip$ \.txt$" > "%LINKS_FILE%"

REM 检查是否找到链接
if %ERRORLEVEL% neq 0 (
    echo No links found. Exiting.
    pause
    exit /b
)

REM 第三步：批量下载文件
echo Downloading files from the extracted links...
wget -i "%LINKS_FILE%" -P "%DOWNLOAD_DIR%"

REM 提示完成
echo Download process completed. Check the "%DOWNLOAD_DIR%" folder.
pause
